\begin{table}[t!]
\scriptsize
\caption{\footnotesize{}Hyperparameter optimization benchmarks used. The loss function is the function value for Branin and Hartmann 6d, perplexity for LDA and misclassification rate for all the others. The runtime column gives the 0.1 and 0.9 quantiles over all function evaluations performed by all optimizers, in minutes. For benchmarks with crossvalidation these are the runtimes for one fold.\label{tab:benchmarks}}
\vspace*{-0.1cm}
\label{tab:experiments}
	\begin{center}
		\begin{tabular}{lccllll}
\toprule
\multirow{2}{*}{\bf Algorithm} &{\bf \#hyp.params} &{\bf continuous/} &\multirow{2}{*}{\bf Dataset} &\multirow{2}{*}{\bf Size (Train/Valid/Test)} &\multirow{2}{*}{\bf Citation} &{\bf Runtime [min]} \\
{ } & {\bf (conditional)} & {\bf discrete} & { } & { } & { } & {\bf q0.1 / q0.9} 

%\multicolumn{1}{l}{\bf Algorithm} &\multicolumn{1}{l}{\bf\tiny \#hyp.params(conditional) } &\multicolumn{1}{l}{\bf\tiny continuous/discrete} &\multicolumn{1}{l}{\bf Dataset} &\multicolumn{1}{l}{\bf Size (Train/Valid/Test)} &\multicolumn{1}{l}{\bf Citation} &\multicolumn{1}{l}{\bf runtime}
\\
\toprule
Branin      &  2(-)   & 2/- & -                   & -                 & \cite{Hed13}      & trivial\\
Hartmann 6d &  6(-)   & 6/- & -                   & -                 & \cite{Hed13}      & trivial\\
\midrule
Log. Reg.   &  4(-)   & 4/- & MNIST               & 50k/10k/10k       & \cite{SnoLarAda12, LecBotHaf98}& $0.3/12.5$\\
%footnotemark\\
LDA ongrid  &  3(-)   & -/3 & Wikipedia articles  & 200k/24560/25k    & \cite{SnoLarAda12, HofBleBac10}& table lookup\\
SVM ongrid  &  3(-)   & -/3 & UniPROBE            & $\approx 20$k/-/$\approx 20$k & \cite{SnoLarAda12, MilEtAl12}& table lookup\\
\midrule
\hpnnet{}        & 14(4)   & 7/7 & MRBI                & 10k/2k/50k        & \cite{BerEtAl11, LarErhCouBerBen07} & $0.9/13.4$\\ % Exactly 4.5 minutes averaged over 2000 runs
%footnotemark\\
\hpnnet{}        & 14(4)   & 7/7 & convex              & 6.5k/1.5k/50k     & \cite{BerEtAl11, LarErhCouBerBen07}& $0.7/16.7$\\ % Exactly 6.75 minutes averaged over 2000 runs
%\hpdbnet{}      & 36(27)   & 19/17   & MRBI              & 10k/2k/50k        & \cite{BerEtAl11, LarErhCouBerBen07}& $\approx 10m$\\
%\footnotemark\\
\hpdbnet{}      & 36(27)   & 19/17   & convex            & 6.5k/1.5k/50k     & \cite{BerEtAl11, LarErhCouBerBen07}& $0.7/46.3$\\ % Exactly 17.2 minutes averaged over 2000 runs
\midrule
%HP-Convnet & 238(235) & 111 /127& Cifar-10          & 50000/-/10000     & \cite{Krizhevsky09}& $ $\\
Auto-WEKA   & 786(784) & 296/490 & convex            & 10 fold cv, 8k Train, 50k Test  & \cite{hall2009weka, ThoEtAl13, LarErhCouBerBen07}& $0.4/15.2$\\
\midrule
Log. Reg. 5CV  &  4(-)   & 4/- & MNIST               & 5 fold cv, 60k Train, 10k Test  & \cite{SnoLarAda12, LecBotHaf98}& $0.3/12.1$\\
\hpnnet{} 5CV       & 14(4)   & 7/7 & MRBI   & 5 fold cv, 12k Train, 50k Test & \cite{BerEtAl11, LarErhCouBerBen07} & $0.9/10.8$\\ % Don't know yet...
\hpnnet{} 5CV       & 14(4)   & 7/7 & convex & 5 fold cv, 8k Train, 50k Test  & \cite{BerEtAl11, LarErhCouBerBen07}& $0.7/10.7$\\ % Exactly 6.65 minutes averaged over 2000 runs
\bottomrule
													\vspace*{-0.4cm}
															\end{tabular}
	\end{center}
\end{table}
%\footnotetext[4]{Run on a cluster with NVIDIA Tesla M2070s cards.}
%\footnotetext[5]{Run on a machine with a NVIDIA GeForce GTX 780.}
%\footnotetext[6]{Run on a cluster with Intel Xeon X5675 chips using two cores.}
%The expensive \hpnnet{} experiments were conducted on NVIDIA Tesla M2070s, the \hpdbnet{} was run on a NVIDIA GeForce GTX 780.
